#!/usr/bin/python
#coding=utf8
import re
import os
import time
import random
#import cPickle
import codecs
import sys
reload(sys)

class Hudong:    
    def __init__(self, bkname):
        self.outfile = open(bkname+'_items.txt', 'w')
        self.outout = open(bkname+'_noit.txt', 'w')
        self.titlepat = re.compile('<title>(.+?)[\-|_].+</title>')
        self.kwpat = re.compile('<meta name="keywords" content="(.+?)"/?>')
        self.aupat = re.compile('<meta name="author" content="(.+?)"/?>')
        self.hrefpat = re.compile('<a .*href="[/|h].+?>([^>]+?)</a>')
        
    def extone(self, filename):
        infile = open(filename, 'r')
        buf = infile.read()
        infile.close()
        pages = buf.split('</html>')
        for page in pages[:-1]:
            self.extdetail(page)

        
    def extdetail(self, page):
        result = []
        try:
            title = self.titlepat.search(page).group(1)
        except:
            title = ''
        result.append('Ti:' + title + '\n')
        if page.find('互动百科尚未收录词条') != -1:
            self.outout.writelines(result)
            return 0
        try:
            kw = self.kwpat.search(page).group(1)
            words = re.split('[,|;]', kw)
            kw = '\t'.join([x for x in words if x.strip()])
        except:
            kw = ''
        result.append('Kw:' + kw + '\n')    
        
        try:
            au = self.aupat.search(page).group(1)
        except:
            au = ''
        result.append('Au:' + au + '\n')       
            
        # Get catagory
        subcata = self.gethrefwords(page, '<fieldset id="catalog">', '</fieldset>')
        result.append('Ml:' + subcata + '\n')  

        # Get tyc
        tyc = self.gethrefwords(page, 'id="show_thesaurus"', '</dl>')
        result.append('TY:' + tyc + '\n')  
        
        # Get belong class
        bc = self.gethrefwords(page, '<p id="thslinkwz"', '</p>')
        result.append('BC:' + bc + '\n')  

        # Get Open class
        oc = self.gethrefwords(page, 'id="show_tag"', '</dl>')
        result.append('OC:' + oc + '\n')  

        # Get BT (target)
        bt = self.gethrefwords(page, '<dt>所属任务', '</dl>')
        result.append('BT:' + bt + '\n')  
             
        # Get RW 
        rw = self.gethrefwords(page, '<h2>相关词条</h2>', '<!--底部开始-->')
        result.append('RW:' + rw + '\n')  
                     
        self.outfile.writelines(result)
        self.outfile.write('\n')
     
    def gethrefwords(self, page, bstr, estr, opstr='class="xgct_more a-r"'):
        bpos = page.find(bstr)
        epos = -1
        if bpos != -1:
            epos = page.find(estr, bpos)
            if epos == -1:
                epos = page.find(opstr, bpos)
#        print "bpos: %d, epos: %d"%(bpos, epos)
        if bpos !=-1 and epos != -1:
            subcata = page[bpos:epos]
        else:
            subcata = ''
        if subcata:
            catas = self.hrefpat.findall(subcata)
            subcata = '\t'.join([x.strip() for x in catas if x.strip()])
        else:
            subcata = ''   
        return subcata     
        
    def doExt(self):
        datadir = '/data/wwqa/baike/src/res/'
        files = [datadir+x for x in os.listdir(datadir) if x.find('hudong')!=-1]
        for afile in files:
            print afile
            self.extone(afile)
            
        self.outfile.close()
        self.outout.close()        
                          
if __name__ == '__main__':
    #startURL = 'http://sports.sina.com.cn'
#    print sys.getdefaultencoding()
#    sys.setdefaultencoding('gbk')
#    print sys.getdefaultencoding()
    if len(sys.argv) != 2:
        print "usage: python exthd.py hudong"
        sys.exit(0)
    se = Hudong(sys.argv[1])
    #se.extone('hudong_0255.txt')
    #se.extone('hudong.html')
    se.doExt()
    print 'ok'
